Python for Data Analysis - Final Projet

user = "samuel" #Values : marius || samuel

import pandas as pd

import seaborn as sns

from matplotlib import pyplot as plt

import matplotlib.colors as plt_colors
import matplotlib.cm as cm
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from textwrap import wrap

from mpl_toolkits.mplot3d import Axes3D

import numpy as np

from datetime import datetime, timedelta

import plotly.graph_objects as go

from plotly.subplots import make_subplots

from gensim.models import KeyedVectors

from sklearn.decomposition import PCA

from sklearn import preprocessing
from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RepeatedKFold
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import mutual_info_regression
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.linear_model import ARDRegression
from sklearn.model_selection import cross_validate,learning_curve
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import confusion_matrix
from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier
from sklearn.metrics import accuracy_score,classification_report
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.ensemble import AdaBoostClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.manifold import TSNE
from sklearn.cluster import KMeans
from sklearn.preprocessing import OneHotEncoder
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import f_classif
from sklearn.feature_selection import mutual_info_classif

from yellowbrick.model_selection import LearningCurve

import plotly.express as px

from collections import OrderedDict

from bokeh.io import output_notebook
from bokeh.models import ColumnDataSource, BasicTicker, ColorBar, LinearColorMapper, PrintfTickFormatter
from bokeh.plotting import figure, show
from bokeh.transform import factor_cmap, transform
output_notebook()

from bokeh.palettes import Spectral5
from bokeh.palettes import PRGn7
from bokeh.palettes import Magma7
from bokeh.palettes import Viridis
from bokeh.palettes import Pastel2_7
from bokeh.palettes import Purples9
from bokeh.palettes import Greens9
from bokeh.palettes import YlOrRd9

from bokeh.palettes import Magma4
from bokeh.palettes import Magma6
from bokeh.palettes import Magma7
from bokeh.palettes import Magma10
from bokeh.palettes import Magma

from wordcloud import WordCloud
from PIL import Image

from missingno import matrix

import tensorflow as tf
from keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Flatten
from tensorflow.keras import optimizers
from tensorflow.keras.layers import Dropout
from keras.utils import to_categorical

import warnings

import pickle
import flask
from flask import Flask, request

vecto_path_marius = r"/content/drive/MyDrive/SWITCH/Python for Data Analysis/Projet Final/DataFrames/GoogleNews-vectors-negative300.bin"
online_news_popularity_path_marius = r"/content/drive/MyDrive/SWITCH/Python for Data Analysis/Projet Final/DataFrames/OnlineNewsPopularityWithAutorsAndTitles.csv"
multi_timeline_path_marius = r"/content/drive/MyDrive/SWITCH/Python for Data Analysis/Projet Final/DataFrames/multiTimeline.csv"
mlogo_path_marius = r"/content/drive/MyDrive/SWITCH/Python for Data Analysis/Projet Final/DataFrames/Mlogo.png"

vecto_path_samuel = r"/content/drive/MyDrive/pyhton/GoogleNews-vectors-negative300.bin"
online_news_popularity_path_samuel = r"/content/drive/MyDrive/pyhton/OnlineNewsPopularityWithAutorsAndTitles.csv"
multi_timeline_path_samuel = r"/content/drive/MyDrive/pyhton/multiTimeline.csv"
mlogo_path_samuel = r"/content/drive/MyDrive/pyhton/Mlogo.png"

if user == "marius":
  vecto_path = vecto_path_marius
  online_news_popularity_path = online_news_popularity_path_marius
  multi_timeline_path = multi_timeline_path_marius
  mlogo_path = mlogo_path_marius
elif user == "samuel":
  vecto_path = vecto_path_samuel
  online_news_popularity_path = online_news_popularity_path_samuel
  multi_timeline_path = multi_timeline_path_samuel
  mlogo_path = mlogo_path_samuel

columns = news.columns
oldColumns = columns
columns = [elem.lstrip() for elem in columns]
dic_cols = {oldColumns[i]:columns[i] for i in range(len(columns))}
news.rename(columns=dic_cols, inplace=True)

scrap = False

if scrap:
    urls = news["url"]
    driver = webdriver.Chrome('chromedriver.exe')

    driver.get(urls[0])
    time.sleep(2)
    element = driver.find_elements(by = By.CSS_SELECTOR, value = 'div[id="onetrust-button-group"]')
    element[0].click()
    authors = []
    titles = []

    authors = []
    titles = []
    for i in range(0, len(urls[i])):
        driver.get(urls[i])
        time.sleep(0.5)
        try : 
            timer1 = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.CSS_SELECTOR,'a[class="underline-link"]')))
            timer2 = WebDriverWait(driver, 2).until(EC.presence_of_element_located((By.CSS_SELECTOR,'h1[class="mt-4 header-100 max-w-5xl "]')))
            name = driver.find_elements(by = By.CSS_SELECTOR, value = 'a[class="underline-link"]')
            title = driver.find_elements(by = By.CSS_SELECTOR, value = 'h1[class="mt-4 header-100 max-w-5xl "]')

            authors.append(name[0].text)
            titles.append(title[0].text)

        except (TimeoutException , NameError):
            authors.append('Nan')
            titles.append('Nan')
    news['Autors'] = authors
    news['titles'] = titles
    df.to_csv('OnlineNewsPopularityWithAutorsAndTitles.csv')

time = [x for x in range(8,732)]

temp = datetime.strptime("27/12/2014",'%d/%m/%Y')
date = []
for i in range(len(time)):
    date.append(temp)
    temp = temp - timedelta(1)

dico = {time[i]:date[i] for i in range(len(time))}
news['date'] = news['timedelta'].map(dico)

days = timeline['Semaine']
count = timeline['Mashable: (Dans tous les pays)']
days1 = []
for day in days:
    days1.append(datetime.strptime(day,'%Y-%m-%d'))

days2 = []
count1 = [] 
days2.append(datetime.strptime('2013-01-03','%Y-%m-%d'))
days2.append(datetime.strptime('2013-01-04','%Y-%m-%d'))
days2.append(datetime.strptime('2013-01-05','%Y-%m-%d'))
count1.append(8.28)
count1.append(8.28)
count1.append(8.28)
for day in days1:
    for i in range(7):
        days2.append(day + timedelta(i))

for coun in count:
    for i in range(7):
        count1.append(coun/7)
count1.append(4.7)
count1.append(4.7)
count1.append(4.7)
count1.append(4.7)
count1.append(4.7)
count1.append(4.7)
days2.append(datetime.strptime('2014-12-22','%Y-%m-%d'))
days2.append(datetime.strptime('2014-12-23','%Y-%m-%d'))
days2.append(datetime.strptime('2014-12-24','%Y-%m-%d'))
days2.append(datetime.strptime('2014-12-25','%Y-%m-%d'))
days2.append(datetime.strptime('2014-12-26','%Y-%m-%d'))
days2.append(datetime.strptime('2014-12-27','%Y-%m-%d'))
dico = {str(days2[i])[:10]:count1[i] for i in range(len(days2))}
news['NbVisit'] = news['date'].astype(str).map(dico)

Class_shares2 = pd.qcut(news.shares,4,
       labels=['very low','low' ,'high', 'very high'])
Class_shares1 = np.where(news['shares'] < 1400, 'low','high')

news['Class_shares1'] = Class_shares1
news['Class_shares2'] = Class_shares2

news = news[news["n_tokens_content"]>0]

for i in range(4, 7): 
    news = news[((news.iloc[:, i] >= 0) & (news.iloc[:, i] <=1))]

summary_13_to_18 = []
for index in range(news.shape[0]):
    if news.iloc[index, 13] == 1:
        summary_13_to_18.append("LifeStyle")
    elif news.iloc[index, 14] == 1:
        summary_13_to_18.append("Entertainment")
    elif news.iloc[index, 15] == 1:
        summary_13_to_18.append("Business")
    elif news.iloc[index, 16] == 1:
        summary_13_to_18.append("Social Media")
    elif news.iloc[index, 17] == 1:
        summary_13_to_18.append("Tech")
    elif news.iloc[index, 18] == 1:
        summary_13_to_18.append("World")
    else:
        summary_13_to_18.append("Others")
news["Chanel"] = summary_13_to_18

for i in range(19, 28):
    news.iloc[:, i] = [x if x>=0 else 0 for x in news.iloc[:, i]]

summary_31_to_37 = []
for index in range(news.shape[0]):
    if news.iloc[index, 31] == 1:
        summary_31_to_37.append("Monday")
    elif news.iloc[index, 32] == 1:
        summary_31_to_37.append("Tuesday")
    elif news.iloc[index, 33] == 1:
        summary_31_to_37.append("Wednesday")
    elif news.iloc[index, 34] == 1:
        summary_31_to_37.append("Thursday")
    elif news.iloc[index, 35] == 1:
        summary_31_to_37.append("Friday")
    elif news.iloc[index, 36] == 1:
        summary_31_to_37.append("Saturday")
    elif news.iloc[index, 37] == 1:
        summary_31_to_37.append("Sunday")
    else:
        summary_31_to_37.append("Unknown")
news["Weekday"] = summary_31_to_37 

#news["shares"] = np.log(news["shares"])

news = news[news["Authors"] != "Nan"]

load = True

if load:
    google_vecto_model = KeyedVectors.load_word2vec_format(vecto_path, binary=True)

news.reset_index(drop = True, inplace = True)

chanels = news.Chanel.unique()
chanels = [x if x != "Social Media" else "SocialMedia" for x in chanels]
vec_chanels = {(chanel):(google_vecto_model[chanel] if chanel != "Others" else [0 for i in range(300)]) for chanel in chanels}
vec_chanels["Social Media"] = vec_chanels.pop("SocialMedia")

all_vect_cols = []
vec_len = len(vec_chanels["Social Media"])
for i in range(vec_len):
    all_vect_cols.append([])

for row in news["Chanel"]:
    for i in range(vec_len):
        all_vect_cols[i].append(vec_chanels[row][i])
        
name_cols_chanels_vec = [f"WChanel {i}" for i in range(vec_len)]
df_chanel_vec = pd.DataFrame(np.transpose(all_vect_cols), columns=name_cols_chanels_vec)
news = pd.concat((news, df_chanel_vec), axis = 1)

export = False

v_news = news.copy()

vecto_cols = [f"WChanel {i}" for i in range(300)]
_=v_news.drop(columns=vecto_cols, inplace = True)

_=v_news.drop(columns=['data_channel_is_lifestyle'], inplace = True)
_=v_news.drop(columns=['data_channel_is_entertainment'], inplace = True)
_=v_news.drop(columns=['data_channel_is_bus'], inplace = True)
_=v_news.drop(columns=['data_channel_is_socmed'], inplace = True)
_=v_news.drop(columns=['data_channel_is_tech'], inplace = True)
_=v_news.drop(columns=['data_channel_is_world'], inplace = True)

_=v_news.drop(columns=['weekday_is_monday'], inplace = True)
_=v_news.drop(columns=['weekday_is_tuesday'], inplace = True)
_=v_news.drop(columns=['weekday_is_wednesday'], inplace = True)
_=v_news.drop(columns=['weekday_is_thursday'], inplace = True)
_=v_news.drop(columns=['weekday_is_friday'], inplace = True)
_=v_news.drop(columns=['weekday_is_saturday'], inplace = True)
_=v_news.drop(columns=['weekday_is_sunday'], inplace = True)
_=v_news.drop(columns=['url'], inplace = True)
_=v_news.drop(columns=['timedelta'], inplace = True)

m_news = news.copy()

_=m_news.drop(columns=['Chanel'], inplace = True)
_=m_news.drop(columns=['Weekday'], inplace = True)
_=m_news.drop(columns=['url'], inplace = True)
_=m_news.drop(columns=['date'], inplace = True)
_=m_news.drop(columns=['Authors'], inplace = True)
_=m_news.drop(columns=['Titles'], inplace = True)
_=m_news.drop(columns=['timedelta'], inplace = True)
_=m_news.drop(columns=['is_weekend'], inplace = True)

m_news.reset_index(drop=True, inplace=True)
m_news = pd.concat((m_news[selected_name_columns], m_news[exclude]), axis = 1)

if export:
    m_news.to_csv("m_news.csv")

lda["Chanel"] = v_news["Chanel"]
map_colors = {
    "Entertainment":"limegreen",
    "Business": "darkturquoise",
    "Tech": "orchid",
    "LifeStyle": "royalblue",
    "World": "gold",
    "Others": "tomato",
    "Social Media": "pink"
}
colors = v_news["Chanel"].map(map_colors)